package com.bds.tjj.snatch;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import javax.annotation.Resource;

import org.apache.commons.collections.ListUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.nutz.dao.Chain;
import org.nutz.dao.Cnd;
import org.nutz.dao.DaoException;
import org.nutz.dao.entity.Record;
import org.nutz.dao.impl.NutDao;
import org.springframework.stereotype.Service;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.bds.tjj.http.HttpBase;
import com.bds.tjj.model.PropIndex;
import com.bds.tjj.model.QuarterAreaData;
import com.bds.tjj.model.QuarterData;
import com.bds.tjj.model.YearAreaData;
import com.bds.tjj.model.YearData;

/**
 * 按年抓取 (年度数据)
 */
@Service
public class StatsYearThread {
	@Resource
	private NutDao dao;
	private static Logger logger = Logger.getLogger(StatsYearThread.class);

	private static String[] cnd_names = { "国内生产总值增长",
			// "城镇登记失业率"
			// "国内生产总值",
			// "化妆美容用品类居民消费价格指数(上年=100)",
			// "化妆品类成交额",
			// "日用百货类商品零售价格指数(上年=100)",
			// "日用品类成交额",
			// "日用杂品类商品零售价格指数(上年=100)",
			// "体育、娱乐用品类成交额",
			// "体育用品零售商品出口额",
			// "体育用品零售商品进口额",
	};
	private static String[] tag1 = {
			// "国民经济核算",
			// "价格指数",
			// "批发和零售业"
			"指标" };
	private static String[] tag2 = { "国民经济核算", "就业人员和工资"
			// "国内生产总值",
			// "居民消费价格分类指数(上年=100)",
			// "商品零售价格分类指数(上年=100)",
			// "限额以上零售业企业基本情况和商品购销存情况",
			// "亿元以上商品交易市场基本情况",
			// "亿元以上商品交易市场摊位分类情况 "
	};
	private static String[] tag3 = { "",
			// "按国民经济行业分限额以上零售业企业商品出口额",
			// "按国民经济行业分限额以上零售业企业商品进口额",
			// "按国民经济行业分限额以上零售业企业商品销售额",
			// "成交额",
			// "零售市场成交额",
			// "医疗保健和个人用品类居民消费价格指数(上年=100)"
			"三次产业对国内生产总值增长的拉动",
			// "城镇登记失业人员及失业率"
	};

	static List<String> big_list;

	public void StartCrawl() throws Exception {
		this.snatchYE0103();
	}

	public void StartCrawlYear() throws Exception {

		// 合并成
		big_list = Arrays.asList(cnd_names);
		List<String> tag1_l = Arrays.asList(tag1);
		List<String> tag2_l = Arrays.asList(tag2);
		List<String> tag3_l = Arrays.asList(tag3);
		big_list = ListUtils.union(big_list, tag1_l);
		big_list = ListUtils.union(big_list, tag2_l);
		big_list = ListUtils.union(big_list, tag3_l);

		String html;
		String html1;
		String html2;
		String html3;
		String html4;
		try {
			Map<String, String> headers = new HashMap<String, String>();
			headers.put("Host", "data.stats.gov.cn");
			headers.put("Origin", "http://data.stats.gov.cn");
			headers.put("Referer", "http://data.stats.gov.cn/easyquery.htm?cn=A01");
			Map<String, String> data2 = new HashMap<String, String>();
			data2.put("id", "zb");
			data2.put("dbcode", "hgnd");
			data2.put("wdcode", "zb");
			data2.put("m", "getTree");
			html = new HttpBase().post("http://data.stats.gov.cn/easyquery.htm", "utf-8", headers, data2);
			html = "{'data':" + html + "}";
			JSONObject c = JSON.parseObject(html);
			JSONArray c10 = c.getJSONArray("data");
			for (int k = 0; k < c10.size(); k++) {
				JSONObject c1010 = c10.getJSONObject(k);
				String pid1 = c1010.getString("id");
				String sub2 = c1010.getString("name");
				// 筛选条件
				if (!big_list.contains(sub2)) {
					continue;
				}
				Map<String, String> data = new HashMap<String, String>();
				data.put("id", pid1);
				data.put("dbcode", "hgnd");
				data.put("wdcode", "zb");
				data.put("m", "getTree");
				html1 = new HttpBase().post("http://data.stats.gov.cn/easyquery.htm", "utf-8", headers, data);
				html1 = "{'data':" + html1 + "}";
				JSONObject a = JSON.parseObject(html1);
				JSONArray a10 = a.getJSONArray("data");
				for (int i = 0; i < a10.size(); i++) {
					JSONObject a1010 = a10.getJSONObject(i);
					String pid = a1010.getString("id");
					String sub3 = a1010.getString("name");
					// 筛选条件
					if (!big_list.contains(sub3)) {
						continue;
					}
					Map<String, String> data1 = new HashMap<String, String>();
					data1.put("id", pid);
					data1.put("dbcode", "hgnd");
					data1.put("wdcode", "zb");
					data1.put("m", "getTree");
					html2 = new HttpBase().post("http://data.stats.gov.cn/easyquery.htm", "utf-8", headers, data1);
					if (html2.equals("[]")) {
						ddd(pid, sub2, sub3, "", "", dao);
					} else {
						html2 = "{'data':" + html2 + "}";
						JSONObject b = JSON.parseObject(html2);
						JSONArray b10 = b.getJSONArray("data");
						for (int j = 0; j < b10.size(); j++) {
							JSONObject b1010 = b10.getJSONObject(j);
							String pid2 = b1010.getString("id");
							String sub4 = b1010.getString("name");
							// 筛选条件
							if (!big_list.contains(sub4)) {
								continue;
							}
							Map<String, String> data3 = new HashMap<String, String>();
							data3.put("id", pid2);
							data3.put("dbcode", "hgnd");
							data3.put("wdcode", "zb");
							data3.put("m", "getTree");
							html4 = new HttpBase().post("http://data.stats.gov.cn/easyquery.htm", "utf-8", headers,
									data3);
							if (html4.equals("[]")) {
								ddd(pid2, sub2, sub3, sub4, "", dao);
							} else {
								html4 = "{'data':" + html4 + "}";
								JSONObject x = JSON.parseObject(html4);
								JSONArray x10 = x.getJSONArray("data");
								for (int y = 0; y < x10.size(); y++) {
									JSONObject x1010 = x10.getJSONObject(y);
									String pid3 = x1010.getString("id");
									String sub5 = x1010.getString("name");
									// 筛选条件
									if (!big_list.contains(sub5)) {
										continue;
									}
									ddd(pid3, sub2, sub3, sub4, sub5, dao);
								}
							}
						}
					}
				}
			}
			System.out.println(html);

		} catch (Exception e) {

			e.printStackTrace();
		}

	}

	public static void main(String args[]) throws Exception {

		StatsYearThread stat = new StatsYearThread();
		stat.StartCrawlYear();

	}

	public void snatchYE0103() throws Exception {
		List<String> big_list = new ArrayList<String>();
		List<Record> list = dao.query("T_PROP_INDEX", Cnd.where("chart_type", "=", "year").and("is_area", "=", "1"));
		for (Record r : list) {
			if (!big_list.contains(r.getString("cn_name"))) {
				big_list.add(r.getString("cn_name"));
			}
		}
		Record area = dao.fetch("T_AREA", Cnd.where("id", "=", 1));
		String areas = area.getString("areas");

		JSONObject areaJson = JSON.parseObject(areas);
		JSONArray nodes = areaJson.getJSONArray("nodes");
		for (int i = 0; i < nodes.size(); i++) {
			JSONObject node = nodes.getJSONObject(i);
			String area_name = node.getString("cname");
			String area_code = node.getString("code");
			String ulr_area = "http://data.stats.gov.cn/easyquery.htm?m=QueryData&dbcode=fsnd&rowcode=zb&colcode=sj&wds=%5B%7B%22wdcode%22%3A%22reg%22%2C%22valuecode%22%3A%22"
					+ area_code
					+ "%22%7D%5D&dfwds=%5B%7B%22wdcode%22%3A%22sj%22%2C%22valuecode%22%3A%22LAST20%22%7D%5D&k1="
					+ new Date().getTime();

			String cookie_area = "_trs_uv=jd1hcdms_6_5v09; JSESSIONID=F1AD03CE377C6C73C1A40FE18AAD8DA5; u=1";
			snatchQ(big_list, ulr_area, cookie_area, "http://data.stats.gov.cn/easyquery.htm?cn=E0103", area_code,
					area_name);

		}

	}

	public void snatchQ(List<String> names, String url, String cookie, String referer, String area_code,
			String area_name) throws Exception {
		Map<String, String> headers = new HashMap<String, String>();
		headers.put("Host", "data.stats.gov.cn");
		headers.put("Referer", referer);
		headers.put("User-Agent",
				"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36");

		// 陶宝的老方法 一个月抓一次用
		headers.put("Cookie", cookie);
		String html3 = new HttpBase().get(url, "utf-8", headers);

		JSONObject d = JSON.parseObject(html3);
		// System.out.println(html3);
		JSONArray d10 = d.getJSONObject("returndata").getJSONArray("wdnodes").getJSONObject(0).getJSONArray("nodes");
		for (int p = 0; p < d10.size(); p++) {
			JSONObject e1010 = d10.getJSONObject(p);
			String code = e1010.getString("code");
			String sub = e1010.getString("name");
			// 筛选条件 (按名字)
			if (!names.contains(sub)) {
				continue;
			}
			JSONArray f10 = d.getJSONObject("returndata").getJSONArray("datanodes");
			for (int o = 0; o < f10.size(); o++) {
				JSONObject f1010 = f10.getJSONObject(o);
				String zbcode = null;
				String date = null;
				String regCode = null;
				JSONArray datas = f1010.getJSONArray("wds");
				for (int j = 0; j < datas.size(); j++) {
					JSONObject ds = datas.getJSONObject(j);
					String wdcode = ds.getString("wdcode");
					if (wdcode.equals("zb")) {
						zbcode = ds.getString("valuecode");
					} else if (wdcode.equals("reg")) {
						regCode = ds.getString("valuecode");
					} else if (wdcode.equals("sj")) {
						date = ds.getString("valuecode");
					}
				}
				if (zbcode.equals(code)) {

					if (StringUtils.isEmpty(area_code) || StringUtils.isEmpty(area_name)) {
						PropIndex prop = dao.fetch(PropIndex.class,
								Cnd.where("cn_name", "=", sub).and("is_area", "=", "0"));

						String strdata = f1010.getJSONObject("data").getString("strdata");

						YearData tmd = new YearData();
						tmd.setCn_name(sub);
						tmd.setCreate_time(new Date());
						tmd.setData(strdata);
						tmd.setEn_name(prop.getEn_name());
						tmd.setYy(Integer.parseInt(date));
						tmd.setProp(prop.getProp());

						try {
							// dao.fastInsert(gi);
							dao.fastInsert(tmd);
							System.out.println(sub + "," + strdata);
						} catch (DaoException e) {
							// 报错就更新 (一般是重复异常)
							YearData mm = dao.fetch(YearData.class,
									Cnd.where("cn_name", "=", sub).and("yy", "=", date));
							if (mm == null) {
								logger.info("月度为空,:" + sub + "," + strdata);
							}
							if (StringUtils.isNotEmpty(strdata)) {
								if (mm.getData() != null) {
									if (!strdata.equals(mm.getData())) {
										dao.update(YearData.class,
												Chain.make("data", strdata).add("update_time", new Date()),
												Cnd.where("cn_name", "=", sub).and("yy", "=", Integer.parseInt(date)));
									} else {
										logger.info("数据相同不更新,:" + sub + "," + strdata);
									}
								} else {
									dao.update(YearData.class,
											Chain.make("data", strdata).add("update_time", new Date()),
											Cnd.where("cn_name", "=", sub).and("yy", "=", Integer.parseInt(date)));
									System.out.println("更新:" + sub + "," + strdata);
								}

							}

						} catch (Exception e) {
							// 别的错误
							System.out.println(e);
						}
					} else {
						PropIndex prop = dao.fetch(PropIndex.class,
								Cnd.where("cn_name", "=", sub).and("is_area", "=", "1"));

						String strdata = f1010.getJSONObject("data").getString("strdata");

						YearAreaData tmd = new YearAreaData();
						tmd.setCn_name(sub);
						tmd.setCreate_time(new Date());
						tmd.setData(strdata);
						tmd.setEn_name(prop.getEn_name());
						tmd.setYy(Integer.parseInt(date));
						tmd.setProp(prop.getProp());
						tmd.setArea_code(area_code);
						tmd.setArea_name(area_name);
						try {
							// dao.fastInsert(gi);
							dao.fastInsert(tmd);
							System.out.println(sub + "," + strdata + "," + area_name);
						} catch (DaoException e) {
							// 报错就更新 (一般是重复异常)
							YearAreaData mm = dao.fetch(YearAreaData.class, Cnd.where("cn_name", "=", sub)
									.and("yy", "=", date).and("area_code", "=", area_code));
							if (mm == null) {
								logger.info("年度为空,:" + sub + "," + strdata);
							}
							if (StringUtils.isNotEmpty(strdata)) {
								if (mm.getData() != null) {
									if (!strdata.equals(mm.getData())) {
										dao.update(YearAreaData.class,
												Chain.make("data", strdata).add("update_time", new Date()),
												Cnd.where("cn_name", "=", sub).and("yy", "=", Integer.parseInt(date))
														.and("area_code", "=", area_code));
									} else {
										logger.info("数据相同不更新,:" + sub + "," + strdata);
									}
								} else {
									dao.update(YearAreaData.class,
											Chain.make("data", strdata).add("update_time", new Date()),
											Cnd.where("cn_name", "=", sub).and("yy", "=", Integer.parseInt(date))
													.and("area_code", "=", area_code));
									System.out.println("更新:" + sub + "," + strdata + "," + area_name);
								}

							}

						} catch (Exception e) {
							// 别的错误
							System.out.println(e);
						}

					}

				}
			}
		}
	}

	public void ddd(String pid, String sub2, String sub3, String sub4, String sub5, NutDao biDao) throws Exception {
		Map<String, String> headers = new HashMap<String, String>();
		headers.put("Host", "data.stats.gov.cn");
		headers.put("Referer", "http://data.stats.gov.cn/easyquery.htm?cn=A01");
		headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
				+ "Chrome/63.0.3239.132 Safari/537.36");
		String url = "http://data.stats.gov.cn/easyquery.htm?m=QueryData&dbcode=hgnd&rowcode=zb&colcode=sj&wds=%5B%5D&dfwds=%5B%7B%22wdcode%22%3A%22zb%22%2C%22valuecode%22%3A%22"
				+ pid + "%22%7D%5D&k1=" + new Date().getTime();
		;

		// Map<String, String> header = new HttpBase().get(url,
		// "utf-8",headers).getHeader();

		// System.out.println(header);

		// 20年
		String url_last20y = "http://data.stats.gov.cn/easyquery.htm?m=QueryData"
				+ "&dbcode=hgnd&rowcode=zb&colcode=sj&wds=%5B%5D&dfwds=%5B%7B%22wdcode"
				+ "%22%3A%22sj%22%2C%22valuecode%22%3A%22LAST20%22%7D%5D&k1=" + new Date().getTime();

		// 获取cookies 这里是增加指标的时候用 因为这种方法可以获取更多数据
		// Response response = Http.get(url, 10000);
		// String cookie = response.getCookie().toString();
		// cookie = cookie + "; u=6"; // 这里获取不到 (实在想抓的话 可以 从浏览器获取 手动加cookie)
		// cookie = "JSESSIONID=CA2A6EE6D3E1FA0C57A087B8489DDEB1; u=1";
		// headers.put("Cookie", cookie);
		// String html3 = HttpBase
		// .get(url_last20y, "utf-8", headers)
		// .getResult();

		// 陶宝的 这个可以获取最新的数据
		String html3 = new HttpBase().get(url, "utf-8", headers);

		JSONObject d = JSON.parseObject(html3);
		JSONArray d10 = d.getJSONObject("returndata").getJSONArray("wdnodes").getJSONObject(0).getJSONArray("nodes");
		for (int p = 0; p < d10.size(); p++) {
			JSONObject e1010 = d10.getJSONObject(p);
			String code = e1010.getString("code");
			String sub = e1010.getString("name");
			// 筛选条件
			if (!Arrays.asList(cnd_names).contains(sub)) {
				continue;
			}
			JSONArray f10 = d.getJSONObject("returndata").getJSONArray("datanodes");
			for (int o = 0; o < f10.size(); o++) {
				JSONObject f1010 = f10.getJSONObject(o);
				String code1 = f1010.getJSONArray("wds").getJSONObject(0).getString("valuecode");
				String date = f1010.getJSONArray("wds").getJSONObject(1).getString("valuecode");
				if (code1.equals(code)) {
					String strdata = f1010.getJSONObject("data").getString("strdata");
					System.out.println(sub2 + "," + sub3 + "," + sub4 + "," + sub + "," + strdata);
					PropIndex prop = dao.fetch(PropIndex.class, Cnd.where("cn_name", "=", sub));
					YearData tyd = new YearData();
					tyd.setCn_name(sub);
					tyd.setCreate_time(new Date());
					tyd.setData(strdata);
					tyd.setEn_name(prop.getEn_name());
					tyd.setYy(Integer.parseInt(date));

					tyd.setProp(prop.getProp());
					// yeargit gi = new yeargit();
					// gi.set分类1("指标");
					// gi.set分类2(sub2);
					// gi.set分类3(sub3);
					// gi.set分类4(sub4);
					// gi.set具体指标(sub);
					// gi.setYearMonth(date);
					// gi.set指标值(strdata);
					// gi.setCreateTime(new Date());
					try {
						// biDao.fastInsert(gi);
						biDao.fastInsert(tyd);
						System.out.println("****");
					} catch (Exception e) {
						System.out.println(e + html3);
					}

				}
			}
		}
	}

}
